# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
from crawler.core.service_owner import *
from crawler.utils.snowflake import *
from crawler.items import *

class EsfOwnerListPipeline(object):
    """
    @author Hongv
    @desc 个人二手房列表页的保存逻辑
    """

    def __init__(self):
        self.esf_service = EsfOwnerService()
        self.snowflake = SnowFlake(workerId=random.randint(0, 31), datacenterId=random.randint(0, 31))

    def process_item(self, item, spider):
        try:
            if not isinstance(item, EsfOfOwner):
                return item
            # 高并发时确保只有一个线程在做处理，避免出现重复数据
            if self.esf_service.task_is_running(item['house_id']):
                item['is_new'] = '0'
                return item
            # is_new  0 不需爬取详情页/1 需要爬取详情页
            item['id'], item['is_new'] = self.snowflake.get_id(), '1'
            item['city_code'] = self.esf_service.get_code('city', item['city_belong'])
            item['web_code'] = self.esf_service.get_code('web', item['web_belong'])
            item['orientation_code'] = self.esf_service.get_code('orientation', item['orientation'])
            if item['total_price'] == '面议': item['unit_price'] = item['total_price'] = '0'
            # 页面显示 1小时前 2天前的 获取具体时间
            published_time, diff_unit, diff_time  = self.esf_service.get_time_by_diff(item['published_time'])
            item['published_time'] = published_time if published_time else item['created_time']
            # 判断数据是否已存在
            result = self.esf_service.get_house_redis(item['house_id'], item['web_code'])
            if result:
                item['is_new'] = '0'
                # 判断数据是否需要更新爬取
                if self.esf_service.check_time_diff(item['created_time'], str(result[3]), diff_unit, diff_time):
                    self.esf_service.update_house_status(item['house_id'], item['web_code'], item['title'], item['published_time'])
                    item['is_new'] = '1'
            else:
                # 数据不存在，保存数据
                self.esf_service.insert_house(item)
                logging.info('\n*** ***\n House {0} Saved !!!< \n*** ***'.format(item['house_id']))
            # 将城市code带入详情页爬取任务中，减少查询
            item['detail_url'] = '{0}<!>{1}'.format(item['detail_url'], item['city_code'])
            return item
        except Exception as e:
            logging.exception('\n*** ***\n ershoufang owner pipeline\n {0}\n Exception {1}\n*** ***'.format(item['house_id'], str(e)))
            item['is_new'] = 'erro'
            return item

    def close_spider(self, spider):
        self.esf_service.dispose()

class EsfOwnerDetailPipeline(object):

    def __init__(self):
        self.esf_service = EsfOwnerService()

    def process_item(self, item, spider):
        if not isinstance(item, EsfOfOwner):
            return item
        try:
            item['district_code'] = self.esf_service.get_code('district', item['district_belong'], sub_str=item['city_code'])
            item['business_code'] = self.esf_service.get_code('spare', item['business_belong'], sub_str=item['district_code'])
            item['property_year_code'] = self.esf_service.get_code('property_year', item.get('property_year','').replace('产权',''))
            item['property_type_code'] = self.esf_service.get_code('property_type', item.get('property_type', ''))
            item['decoration_code'] = self.esf_service.get_code('decoration', item.get('decoration', ''))
            item['architecturaltype_code'] = self.esf_service.get_code('architecturaltype', item.get('architecturaltype', '').replace('普通',''))
            item['updated_time'], item['img_count'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), len(json.loads(item['picture']))
            self.esf_service.update_house(item)
            self.esf_service.end()
            logging.info('\n*** ***\n House {0} Saved !!!< \n*** ***'.format(item['house_id']))
            return item
        except Exception as e:
            logging.exception('\n*** ***\n ershoufang owner pipeline\n {0}\n Exception {1}\n*** ***'.format(item['house_id'], str(e)))
            return item

    def close_spider(self,spider):
        self.esf_service.dispose()

class RentOwnerListPipeline(object):
    """
    @author Hongv
    @desc 个人出租房列表页的保存逻辑
    """
    def __init__(self):
        self.rent_service = RentOwnerService()
        self.snowflake = SnowFlake(workerId=random.randint(0, 31), datacenterId=random.randint(0, 31))

    def process_item(self, item, spider):
        try:
            if not isinstance(item, RentOfOwner):
                return item
            # 高并发时确保只有一个线程在做处理，避免出现重复数据
            if self.rent_service.task_is_running(item['house_id']):
                item['is_new'] = '0'
                return item

            item['id'], item['is_new'] = self.snowflake.get_id(), '1'
            item['city_code'] = self.rent_service.get_code('city', item['city_belong'])
            item['published_time'] = self.rent_service.stamp13_to_str(int(item['published_time']))
            item['web_code'] = self.rent_service.get_code('web', item['web_belong'])
            item['lease_type_code'] = self.rent_service.get_code('lease_type', item['lease_type'])

            # 判断数据是否已存在
            result = self.rent_service.get_house_redis(item['house_id'], item['web_code'])
            if result:
                item['is_new'] = '0'
                if str(result[3]) != item['published_time']:
                    self.rent_service.update_house_status(item['house_id'], item['web_code'], item['published_time'])
                    item['is_new'] = '1'
            else:
                self.rent_service.insert_house(item)
                logging.info('\n*** ***\n House {0} Saved !!!< \n*** ***'.format(item['house_id']))
            item['detail_url'] = '%s<!>%s' % (item['detail_url'], item['city_code'])
            return item
        except Exception as e:
            logging.exception('\n*** ***\n chuzu owner pipeline\n {0}\n Exception {1}\n*** ***'.format(item['house_id'], str(e)))
            item['is_new'] = 'erro'
            return item

    def close_spider(self,spider):
        self.rent_service.dispose()

class RentOwnerDetailPipeline(object):
    def __init__(self):
        self.rent_service = RentOwnerService()

    def process_item(self, item, spider):
        if not isinstance(item, RentOfOwner):
            return item

        try:
            item['district_code'] = self.rent_service.get_code('district', item['district_belong'], sub_str=item['city_code'])
            item['business_code'] = self.rent_service.get_code('spare', item['business_belong'], sub_str=item['district_code'])
            item['updated_time'], item['img_count'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), len(json.loads(item['picture']))
            item['decoration_code'] = self.rent_service.get_code('decoration', item.get('decoration', ''))
            item['orientation_code'] = self.rent_service.get_code('orientation', item['orientation'])
            item['pay_type_code'] = self.rent_service.get_code('pay_type', item['pay_type'])
            self.rent_service.update_house(item)
            self.rent_service.end()
            logging.info('\n*** ***\n House {0} Saved !!!< \n*** ***'.format(item['house_id']))
            return item
        except Exception as e:
            logging.exception('\n*** ***\n chuzu owner pipeline\n {0}\n Exception {1}\n*** ***'.format(item['house_id'], str(e)))
            return item

    def close_spider(self,spider):
        self.rent_service.dispose()

class EsfOwnerRedisPipeline(object):
    """
    @author Hongv
    @desc 存储详情页链接
    """

    def __init__(self):
        self.esf_service = EsfOwnerService()

    def process_item(self, item, spider):
        if item['is_new'] == '0':
            return item
        if self.esf_service.get_link_redis(item['detail_url']) or item['is_new'] == 'erro':
            raise DropItem('%s id exists!!' % (item['detail_url']))
        else:
            self.esf_service.insert_info_redis(item['detail_url'])
        return item

class RentOwnerRedisPipeline(object):
    """
    @author Hongv
    @desc 存储详情页链接
    """

    def __init__(self):
        self.rent_service = RentOwnerService()

    def process_item(self, item, spider):
        if item['is_new'] == '0':
            return item
        if self.rent_service.get_link_redis(item['detail_url']) or item['is_new'] == 'erro':
            raise DropItem('%s id exists!!' % (item['detail_url']))
        else:
            self.rent_service.insert_info_redis(item['detail_url'])
        return item